homeassistant-ai
diff --git a/‎.claude/skills/bat-story-eval/SKILL.md‎
Lines changed: 10 additions & 2 deletions b/‎.claude/skills/bat-story-eval/SKILL.md‎
Lines changed: 10 additions & 2 deletions
diff --git a/‎.claude/skills/bat-story-eval/references/evaluation-protocol.md‎
Lines changed: 10 additions & 1 deletion b/‎.claude/skills/bat-story-eval/references/evaluation-protocol.md‎
Lines changed: 10 additions & 1 deletion
diff --git a/‎.github/workflows/pr.yml‎
Lines changed: 1 addition & 1 deletion b/‎.github/workflows/pr.yml‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎packaging/mcpb/generate_manifest.py‎
Lines changed: 1 addition & 1 deletion b/‎packaging/mcpb/generate_manifest.py‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎pyproject.toml‎
Lines changed: 10 additions & 2 deletions b/‎pyproject.toml‎
Lines changed: 10 additions & 2 deletions
diff --git a/‎src/ha_mcp/config.py‎
Lines changed: 13 additions & 7 deletions b/‎src/ha_mcp/config.py‎
Lines changed: 13 additions & 7 deletions
diff --git a/‎tests/src/unit/test_config.py‎
Lines changed: 82 additions & 19 deletions b/‎tests/src/unit/test_config.py‎
Lines changed: 82 additions & 19 deletions
@@ -131,7 +131,11 @@ uv run python tests/uat/stories/scripts/ha_query.py \
   --agent <agent> \
   "Does an automation with alias 'Sunset Porch Light' exist?"
 ```
-Record each answer as **confirmed** / **denied** / **unclear**.
+Record each answer as **confirmed** / **denied** / **unclear**. A non-zero exit
+from `ha_query.py` means the query itself failed (the output carries an
+`[exit N]` marker; `[exit 124]` is a timeout) — that is not one of the three
+outcomes; re-run it, and if it keeps failing record the story as `unverified`
+(Step 5) rather than scoring it. See `references/evaluation-protocol.md`.
 
 Run remaining pre-built stories on the same container:
 ```bash
@@ -256,9 +260,11 @@ For each story+agent:
 
 Append eval results as NEW lines (never modify existing):
 ```python
-record["eval_score"] = "pass"  # or "partial" or "fail"
+record["eval_score"] = "pass"  # or "partial", "fail", or "unverified"
 record["eval_notes"] = "Entity created, triggers verified"
 record["eval_trend"] = "stable"  # or "new", "improved", "decreased"
+# "unverified" is for a story whose verification query itself failed — it is
+# not a result, so it carries no trend and is not compared to the baseline.
 ```
 
 ## Step 6: Report
@@ -353,6 +359,8 @@ Flag >5% total size increase (directly impacts token cost per turn).
 | `tests/uat/stories/scripts/ha_query.py` | Query live HA via agent+MCP for verification |
 | `tests/uat/stories/catalog/s*.yaml` | Pre-built story definitions |
 | `local/uat-results.jsonl` | Historical results (gitignored) |
+| `references/evaluation-protocol.md` | Scoring rules, verification questions, cross-agent checks |
+| `references/regression-protocol.md` | Regression classification and flaky handling |
 
 ## Important Notes
 
 
@@ -29,7 +29,16 @@ uv run python tests/uat/stories/scripts/ha_query.py \
   "Does an automation with alias 'Sunset Porch Light' exist? Show its triggers and actions."
 ```
 
-Evaluate the response:
+Check the exit code first. `ha_query.py` exits non-zero when the agent CLI
+itself failed, and prints `[exit N]` (plus stderr, when there is any) after
+whatever text the CLI managed to produce — `[exit 124]` for a query that hung
+past its timeout. The one non-zero exit without a marker is `Error: <agent>
+CLI not found`, which means the agent is not installed: fix the environment
+rather than re-running. A failed query is **not a measurement**: do not score
+it as any of the three outcomes below — re-run it, and if it keeps failing,
+record the story as `unverified` (see Step 5 of SKILL.md) and say why.
+
+Only when the query exited 0, evaluate the response:
 - **Confirmed**: The answer clearly confirms the expected outcome
 - **Denied**: The answer clearly shows the expected outcome did NOT happen
 - **Unclear**: The answer is ambiguous or incomplete
 
@@ -270,7 +270,7 @@ jobs:
       run: uv sync --dev
 
     - name: Run ruff check
-      run: uv run ruff check src/ tests/ custom_components/ homeassistant-addon/ homeassistant-addon-webhook-proxy/ homeassistant-addon-webhook-proxy-dev/ scripts/
+      run: uv run ruff check src/ tests/ custom_components/ homeassistant-addon/ homeassistant-addon-webhook-proxy/ homeassistant-addon-webhook-proxy-dev/ packaging/ scripts/
 
     - name: Run ruff format check on changed Python files
       run: |
 
@@ -12,7 +12,7 @@
 def _get_docstring_description(node: ast.AsyncFunctionDef) -> str:
     """Get the first line of a function's docstring for use as a description."""
     docstring = ast.get_docstring(node) or ""
-    return docstring.split("\n")[0].strip() if docstring else ""
+    return docstring.split("\n", maxsplit=1)[0].strip() if docstring else ""
 
 
 def _extract_title_and_decorator_description(
 
@@ -186,8 +186,16 @@ ignore = [
 # homeassistant-addon-webhook-proxy-dev/ only, per the dev-first/promote-only
 # flow (allow-stable-edit is for stable-only hotfixes, not routine sweeps —
 # see homeassistant-addon-webhook-proxy/AGENTS.md "Dev-first, promote-only").
-# Remove this ignore once the next promote carries the fix into stable.
-"homeassistant-addon-webhook-proxy/**" = ["PLC0207", "PLR5501"]
+# The -dev entries name where a reset-dev-from-stable run puts those same two
+# stable files (start.py stays put, mcp_proxy/ is renamed to mcp_proxy_dev/),
+# so the auto-opened reset PR lints clean. Scoped per file and per rule so
+# both rules stay live everywhere else in both trees — the -dev tree is where
+# all new webhook-proxy code is authored. Remove all four once the next
+# promote carries the fixes into stable.
+"homeassistant-addon-webhook-proxy/start.py" = ["PLR5501"]
+"homeassistant-addon-webhook-proxy/mcp_proxy/__init__.py" = ["PLC0207"]
+"homeassistant-addon-webhook-proxy-dev/start.py" = ["PLR5501"]
+"homeassistant-addon-webhook-proxy-dev/mcp_proxy_dev/__init__.py" = ["PLC0207"]
 # C901 is enforced repo-wide with no per-file exemptions (the grandfathered
 # list was fully cleared by issue #925). Do NOT add "path" = ["C901"] entries
 # here — extract helpers to bring the function below the threshold instead:
 
@@ -33,15 +33,13 @@
 # Support for different environment files via HAMCP_ENV_FILE
 env_file = os.getenv("HAMCP_ENV_FILE", ".env")
 env_path = project_root / env_file
+if not env_path.exists():
+    # Fallback to default .env
+    env_path = project_root / ".env"
 
-# Load the specified environment file (silently, since env vars may come from other sources)
+# Load the environment file (silently, since env vars may come from other sources)
 if env_path.exists():
     load_dotenv(env_path)
-else:
-    # Fallback to default .env
-    default_env_path = project_root / ".env"
-    if default_env_path.exists():
-        load_dotenv(default_env_path)
 
 
 class Settings(BaseSettings):
@@ -565,7 +563,15 @@ def _lenient_sidecar_pin_port(cls, v: object) -> int:
         return port
 
     model_config = SettingsConfigDict(
-        env_file=".env", env_file_encoding="utf-8", case_sensitive=False, extra="allow"
+        # Absolute, and the same file load_dotenv already resolved above. A
+        # relative ".env" here would be a second, independent read that
+        # pydantic-settings resolves against the process's working directory —
+        # so HAMCP_ENV_FILE would not govern it, and a stray .env in whatever
+        # directory the server was launched from would silently supply values.
+        env_file=str(env_path),
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="allow",
     )
 
 
 
@@ -115,6 +115,25 @@ def test_enable_strict_mandatory_bps_env_var_coercion_rejected(env_value):
         )
 
 
+def test_settings_ignores_an_env_file_in_the_working_directory(tmp_path, monkeypatch):
+    """The project ``.env`` is the only one read — never the cwd's.
+
+    ``model_config``'s ``env_file`` is a second dotenv read, independent of the
+    ``load_dotenv`` at import. Left relative it resolves against the process's
+    working directory, so launching the server from a directory holding an
+    unrelated ``.env`` silently supplied values for any matching field name
+    (``timeout``, ``debug``, ``log_level`` … are common in other projects'
+    files), and ``HAMCP_ENV_FILE`` could not override it.
+    """
+    (tmp_path / ".env").write_text("HOMEASSISTANT_URL=http://from-cwd:8123\n")
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.delenv("HOMEASSISTANT_URL", raising=False)
+
+    from ha_mcp.config import Settings
+
+    assert Settings().homeassistant_url != "http://from-cwd:8123"
+
+
 def test_read_only_mode_disabled_by_default():
     """read_only_mode defaults to False (opt-in safety toggle, #1569)."""
     from ha_mcp.config import Settings
@@ -161,18 +180,43 @@ def test_read_only_mode_env_var_coercion_rejected(env_value):
 class TestConfigErrorHandling:
     """Test configuration error handling and user-friendly messages."""
 
-    def test_missing_env_vars_shows_friendly_message(self):
+    @pytest.fixture
+    def child_env(self, tmp_path):
+        """Child-process env for half of the isolation from a developer ``.env``.
+
+        Dropping the credential vars from the child environment is not enough
+        on a machine with the documented local ``.env``: the subprocess would
+        start up fine and never print the message under test. There are two
+        independent readers of that file and each needs its own mitigation.
+
+        This fixture handles the first: ``ha_mcp.config`` runs ``load_dotenv``
+        on ``<project_root>/.env`` by absolute path at import. Only an
+        ``HAMCP_ENV_FILE`` that *exists* wins over it — a path that does not
+        exist falls straight back to ``.env``, so an empty file is the lever.
+
+        The second is ``Settings.model_config``'s ``env_file=".env"``, which
+        pydantic-settings resolves against the child's **cwd** and which
+        ``HAMCP_ENV_FILE`` cannot reach. Every test here therefore also passes
+        ``cwd=tmp_path``. Both are required; neither alone isolates the child.
+        """
+        empty_env_file = tmp_path / "empty.env"
+        empty_env_file.write_text("")
+        env = os.environ.copy()
+        env["HAMCP_ENV_FILE"] = str(empty_env_file)
+        return env
+
+    def test_missing_env_vars_shows_friendly_message(self, child_env, tmp_path):
         """When HOMEASSISTANT_URL and TOKEN are missing, show friendly error."""
         # Run ha-mcp without any env vars set
-        env = os.environ.copy()
+        env = child_env
         # Remove any HA env vars that might be set
         env.pop("HOMEASSISTANT_URL", None)
         env.pop("HOMEASSISTANT_TOKEN", None)
-        env.pop("HAMCP_ENV_FILE", None)
 
         result = subprocess.run(
             [sys.executable, "-m", "ha_mcp"],
             env=env,
+            cwd=tmp_path,
             capture_output=True,
             text=True,
             timeout=30,
@@ -185,82 +229,101 @@ def test_missing_env_vars_shows_friendly_message(self):
         # Should show friendly message, not raw stacktrace
         stderr = result.stderr
         assert "Configuration Error" in stderr
-        assert "HOMEASSISTANT_URL" in stderr
-        assert "HOMEASSISTANT_TOKEN" in stderr
+        # The two-space-dash prefix is the dynamic "what is actually missing"
+        # list. The static body of the banner names both variables regardless
+        # of which one is missing, so a bare name match proves nothing.
+        assert "  - HOMEASSISTANT_URL" in stderr
+        assert "  - HOMEASSISTANT_TOKEN" in stderr
         assert "Long-Lived Access Tokens" in stderr
         assert "github.com/homeassistant-ai/ha-mcp" in stderr
 
         # Should NOT show raw pydantic validation error
         assert "pydantic_core._pydantic_core.ValidationError" not in stderr
         assert "Field required [type=missing" not in stderr
 
-    def test_missing_only_url_shows_that_var(self):
+    def test_missing_only_url_shows_that_var(self, child_env, tmp_path):
         """When only HOMEASSISTANT_URL is missing, show that in message."""
-        env = os.environ.copy()
+        env = child_env
         env.pop("HOMEASSISTANT_URL", None)
-        env.pop("HAMCP_ENV_FILE", None)
         env["HOMEASSISTANT_TOKEN"] = "test_token_value"
 
         result = subprocess.run(
             [sys.executable, "-m", "ha_mcp"],
             env=env,
+            cwd=tmp_path,
             capture_output=True,
             text=True,
             timeout=30,
             check=False,
         )
 
         assert result.returncode != 0
-        assert "HOMEASSISTANT_URL" in result.stderr
+        # Match the dynamic missing-list entry, and require the variable that
+        # IS set to be absent from it — the static banner body names both.
+        assert "  - HOMEASSISTANT_URL" in result.stderr
+        assert "  - HOMEASSISTANT_TOKEN" not in result.stderr
 
-    def test_missing_only_token_shows_that_var(self):
+    def test_missing_only_token_shows_that_var(self, child_env, tmp_path):
         """When only HOMEASSISTANT_TOKEN is missing, show that in message."""
-        env = os.environ.copy()
+        env = child_env
         env.pop("HOMEASSISTANT_TOKEN", None)
-        env.pop("HAMCP_ENV_FILE", None)
         env["HOMEASSISTANT_URL"] = "http://test.local:8123"
 
         result = subprocess.run(
             [sys.executable, "-m", "ha_mcp"],
             env=env,
+            cwd=tmp_path,
             capture_output=True,
             text=True,
             timeout=30,
             check=False,
         )
 
         assert result.returncode != 0
-        assert "HOMEASSISTANT_TOKEN" in result.stderr
+        # Match the dynamic missing-list entry, and require the variable that
+        # IS set to be absent from it — the static banner body names both.
+        assert "  - HOMEASSISTANT_TOKEN" in result.stderr
+        assert "  - HOMEASSISTANT_URL" not in result.stderr
 
-    def test_no_env_file_warning_removed(self):
+    def test_no_env_file_warning_removed(self, child_env, tmp_path):
         """No warning should be shown when .env file is missing."""
-        env = os.environ.copy()
+        env = child_env
         env.pop("HOMEASSISTANT_URL", None)
         env.pop("HOMEASSISTANT_TOKEN", None)
-        env.pop("HAMCP_ENV_FILE", None)
 
         result = subprocess.run(
             [sys.executable, "-m", "ha_mcp"],
             env=env,
+            cwd=tmp_path,
             capture_output=True,
             text=True,
             timeout=30,
             check=False,
         )
 
+        # Liveness anchor: the run has to reach the credential check and fail
+        # there. Without this, a startup crash (import error, packaging break)
+        # never prints the warning either and passes the assertion below. The
+        # two-space-dash entries are the dynamic missing-vars list, which no
+        # other startup banner emits.
+        assert result.returncode != 0
+        assert "Configuration Error" in result.stderr
+        assert "  - HOMEASSISTANT_URL" in result.stderr
+        assert "  - HOMEASSISTANT_TOKEN" in result.stderr
+
         # Should NOT contain the old noisy warning
         combined_output = result.stdout + result.stderr
         assert "[ENV] WARNING: No environment file found" not in combined_output
 
-    def test_smoke_test_still_works(self):
+    def test_smoke_test_still_works(self, child_env, tmp_path):
         """Smoke test should work with dummy credentials."""
-        env = os.environ.copy()
         # Smoke test sets its own dummy credentials
-        env.pop("HAMCP_ENV_FILE", None)
+        env = child_env
 
         result = subprocess.run(
             [sys.executable, "-m", "ha_mcp", "--smoke-test"],
             env=env,
+            cwd=tmp_path,
             capture_output=True,
             text=True,
             timeout=60,