fix: tolerate setuptools editable-finder KeyError on Python 3.14 bring-up (#1989)

On Python 3.14 with `homeassistant` installed as a setuptools editable package
(official HA container image / HA OS), CPython's `PathFinder.invalidate_caches()`
raises `KeyError` at its `del sys.path_importer_cache[name]` line — the synthetic
`__editable__.<dist>.finder.__path_hook__` placeholder is not an absolute path,
so CPython takes the `del` branch on a key an earlier iteration already removed
(a CPython bug; should be a `pop`). That aborts `importlib.invalidate_caches()`
and crashes `_installed_ha_mcp_version()` on the `async_start()` bring-up path,
so the in-process server never comes up and every tool fails (#1891, #1985).
Deterministic on Python 3.14; invisible on 3.13 / non-editable installs.

The four `importlib.invalidate_caches()` call sites route through
`_safe_invalidate_caches()`: on that `KeyError` it prunes the stale dead/relative
`sys.path_importer_cache` entries with `pop` (the offending placeholder among
them), then re-runs `importlib.invalidate_caches()` so CPython completes its full
sweep — path-entry finders, namespace-path epoch, and metadata. The retry is
guarded (a second KeyError from a concurrent re-add is tolerated, best-effort,
never re-crashes bring-up); recovery is logged at WARNING for diagnosis. The
broad `except KeyError` is deliberate — the same CPython bug also fires via the
`finder is None` branch with an absolute-path key, so narrowing by key would
re-raise that variant.

Covered by unit tests (`TestSafeInvalidateCaches`) and an in-container e2e probe
that injects the exact KeyError against the real component in a live HA
container. Rides under the pending 1.2.3 component version.

Closes #1985.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: kingpanther13

custom_components/ha_mcp_tools/embedded_server.py

@@ -1568,6 +1568,64 @@ def _prune_and_check_importing_workers() -> bool:
 _CACHED_IMPORT_VERSION: str | None = None
 
 
+def _safe_invalidate_caches() -> None:
+    """Run ``importlib.invalidate_caches()``, completing it if a finder breaks.
+
+    On Python 3.14 with ``homeassistant`` installed as a setuptools *editable*
+    package (the official HA container image), ``PathFinder.invalidate_caches()``
+    raises ``KeyError`` at its ``del sys.path_importer_cache[name]`` line: the
+    synthetic ``__editable__.<dist>.finder.__path_hook__`` placeholder is not an
+    absolute path, so CPython takes the ``del`` branch on a key an earlier
+    iteration already removed — a CPython 3.14 bug (the ``del`` should be a
+    ``pop``). That aborts ``importlib.invalidate_caches()`` partway and, before
+    this guard, crashed in-process server bring-up on every boot (issues #1891,
+    #1985).
+
+    On that ``KeyError`` we do CPython's own cleanup ourselves — prune the dead /
+    relative ``sys.path_importer_cache`` entries with ``pop`` instead of the
+    buggy ``del`` (the stale placeholder that trips the sweep is among them) —
+    then re-run ``importlib.invalidate_caches()``. With the offending entries
+    gone the retry completes CPython's full sweep itself: every live path-entry
+    finder invalidated, the namespace-path epoch advanced, and the metadata
+    finder refreshed. Delegating the second pass keeps us off private internals
+    (no ``_NamespacePath`` / ``_path_isabs`` poking) and faithful to whatever the
+    running Python's ``invalidate_caches`` does. Recovery only ever runs on the
+    broken 3.14 path (the top-level call succeeds everywhere else); every
+    non-``KeyError`` still propagates.
+
+    The retry is itself guarded: a concurrent import on another HA-core thread
+    could re-add a stale placeholder in the window between the prune and the
+    retry, so a *second* ``KeyError`` is tolerated (logged, best-effort) rather
+    than re-raised — a partial cache refresh must never re-crash bring-up, which
+    is the whole point of this helper. The recovery is logged at WARNING (it
+    recurs on every version check on an affected install), so it is visible for
+    diagnosis rather than a silent workaround.
+    """
+    try:
+        importlib.invalidate_caches()
+        return
+    except KeyError as err:
+        _LOGGER.warning(
+            "importlib.invalidate_caches() raised KeyError from a broken "
+            "(setuptools editable / Python 3.14) finder; pruning stale "
+            "sys.path_importer_cache entries and retrying: %s",
+            err,
+        )
+    for name in list(sys.path_importer_cache):
+        if sys.path_importer_cache.get(name) is None or not os.path.isabs(name):
+            sys.path_importer_cache.pop(name, None)
+    try:
+        importlib.invalidate_caches()
+    except KeyError as err:
+        _LOGGER.warning(
+            "importlib.invalidate_caches() still raised KeyError after pruning "
+            "stale sys.path_importer_cache entries; continuing with a best-effort "
+            "cache state (a concurrent import may have re-added the placeholder): "
+            "%s",
+            err,
+        )
+
+
 def _purge_ha_mcp_modules() -> None:
     """Drop every cached ``ha_mcp`` module so the next import loads fresh code.
 
@@ -1595,7 +1653,7 @@ def _purge_ha_mcp_modules() -> None:
         return
     for name in purged:
         sys.modules.pop(name, None)
-    importlib.invalidate_caches()
+    _safe_invalidate_caches()
     _LOGGER.debug("Purged %d cached ha_mcp module(s) before worker start", len(purged))
 
 
@@ -1608,7 +1666,7 @@ def _installed_ha_mcp_version(preferred_dist: str | None = None) -> str | None:
     provided, checks that channel first so stale metadata from a failed
     best-effort conflicting uninstall cannot mask the package just installed.
     """
-    importlib.invalidate_caches()
+    _safe_invalidate_caches()
     # Metadata alone is not proof: a channel switch's best-effort uninstall
     # can leave ORPHANED .dist-info whose files are gone (the shared ha_mcp/
     # tree belongs to whichever dist installed last). Require the import
@@ -1631,7 +1689,7 @@ def _dist_installed(dist_name: str) -> bool:
 
     Invalidates the import caches first so a just-completed (un)install is seen.
     """
-    importlib.invalidate_caches()
+    _safe_invalidate_caches()
     try:
         importlib.metadata.version(dist_name)
     except importlib.metadata.PackageNotFoundError:
@@ -1648,7 +1706,7 @@ def _installed_dist_version(dist_name: str) -> str | None:
     the auto-update check compares the newest PyPI build against the version of
     the channel actually installed.
     """
-    importlib.invalidate_caches()
+    _safe_invalidate_caches()
     try:
         return importlib.metadata.version(dist_name)
     except importlib.metadata.PackageNotFoundError:

tests/src/e2e/basic/test_backend_dispatch_smoke.py

@@ -85,8 +85,8 @@
     # Entries below are CI-observed item counts, bumped only for intentional
     # marker-gated additions rather than runtime skips.
     "container": 72,  # was 71; +1 Puppet-management test (haos_only + inaddon_only)
-    "haos": 45,  # was 39; +1 Puppet-management test and +5 screenshot sidecar tests
-    "haos_inaddon": 72,  # was 67; +5 screenshot sidecar tests (container_only)
+    "haos": 46,  # was 45; +1 py3.14 invalidate_caches recovery e2e (container_only)
+    "haos_inaddon": 73,  # was 72; +1 py3.14 invalidate_caches recovery e2e (container_only)
     # Embedded backend (#1527, E2E_BACKEND=embedded). Skips exactly the container
     # lane's marker-skips PLUS two embedded-specific additions:
     #   - haos_only + inaddon_only tests skip on embedded just like on container
@@ -100,7 +100,7 @@
     # 1) + not_on_embedded 2 = 100. Initially set to 115 as a buffer for
     # parametrize item-inflation; round 6 (run 28709196071) observed the exact
     # item count and the entry below is pinned to it.
-    "embedded": 128,  # was 127; +1 Puppet-management test (haos_only + inaddon_only)
+    "embedded": 129,  # was 128; +1 py3.14 invalidate_caches recovery e2e (not_on_embedded)
     # HAOS embedded backend (#1527, HAOS_TEST_MODE=embedded). A HAOS lane, so it
     # skips the SAME set as the external HAOS lane (container_only + inaddon_only)
     # PLUS two haos_embedded-specific additions:
@@ -118,7 +118,7 @@
     # lanes show (haos def 30 → ~35 observed; haos_inaddon def 50 → ~58) gives
     # ~84; initially set to 90 with a small buffer, and round 8 observed
     # exactly 90 — the entry below is pinned to the observed count.
-    "haos_embedded": 102,  # was 96; +1 Puppet-management test and +5 screenshot sidecar tests
+    "haos_embedded": 103,  # was 102; +1 py3.14 invalidate_caches recovery e2e (container_only)
 }
 
 

tests/src/e2e/workflows/embedded/test_embedded_server.py

@@ -82,6 +82,40 @@
 _INITIAL_STATE = _REPO_ROOT / "tests" / "initial_test_state"
 _INTEGRATION_SRC = _REPO_ROOT / "custom_components" / "ha_mcp_tools"
 
+# Run INSIDE the live HA container (a separate ``python3`` process, so HA itself
+# is never touched) to prove _safe_invalidate_caches() recovers from the
+# Python-3.14 setuptools editable-finder KeyError (#1891/#1985) on the REAL
+# component code + container interpreter. The container here is Python 3.13, so
+# the crash can't occur naturally — we inject a meta-path finder that raises the
+# exact KeyError while the stale placeholder is cached (what CPython 3.14's
+# PathFinder `del` trips on). With the fix, the helper prunes the placeholder and
+# the retry succeeds; revert the fix and the KeyError propagates → exit != 0.
+_INVALIDATE_RECOVERY_PROBE = """
+import sys
+sys.path.insert(0, "/config")
+from custom_components.ha_mcp_tools.embedded_server import _safe_invalidate_caches
+
+PLACEHOLDER = "__editable__.homeassistant-2026.7.2.finder.__path_hook__"
+
+
+class _EditableFinderKeyErrorSim:
+    def find_spec(self, fullname, path=None, target=None):
+        return None
+
+    def invalidate_caches(self):
+        if PLACEHOLDER in sys.path_importer_cache:
+            raise KeyError(PLACEHOLDER)
+
+
+sys.path_importer_cache[PLACEHOLDER] = None
+sys.meta_path.insert(0, _EditableFinderKeyErrorSim())
+
+_safe_invalidate_caches()
+
+assert PLACEHOLDER not in sys.path_importer_cache, "stale placeholder not pruned"
+print("RECOVERY_OK")
+"""
+
 
 def _docker_available() -> bool:
     try:
@@ -258,10 +292,12 @@ def _initialize(base_url: str) -> tuple[bool, str | None]:
 def embedded_ha():
     """Boot a dedicated HA container running the in-process MCP server entry.
 
-    Yields ``(base_url, session_id, config_path)`` once the in-process MCP
-    server has installed itself, started, and registered its ingress webhook.
+    Yields ``(base_url, session_id, config_path, container)`` once the in-process
+    MCP server has installed itself, started, and registered its ingress webhook.
     ``config_path`` is the bind-mounted /config dir — the LLM-API test reads
     ``home-assistant.log`` from it to prove the registration ran inside HA.
+    ``container`` is the testcontainers handle — the invalidate-caches recovery
+    test ``exec_run``s an in-container probe through it.
     """
     if not _docker_available():
         pytest.skip("Docker is not available for the embedded-server e2e")
@@ -320,15 +356,15 @@ def embedded_ha():
                 "in-process MCP server did not become reachable via its webhook within "
                 f"{_READY_TIMEOUT_S}s. Container logs:\n{logs}"
             )
-        yield base_url, session_id, config_path
+        yield base_url, session_id, config_path, container
     finally:
         with contextlib.suppress(Exception):
             container.stop()
 
 
 class TestEmbeddedServerEndToEnd:
     def test_initialize_and_list_tools(self, embedded_ha):
-        base_url, session_id, _config = embedded_ha
+        base_url, session_id, _config, _container = embedded_ha
         resp = _mcp_post(
             base_url,
             {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}},
@@ -345,7 +381,7 @@ def test_initialize_and_list_tools(self, embedded_ha):
         assert "ha_get_state" in names
 
     def test_read_only_tool_call(self, embedded_ha):
-        base_url, session_id, _config = embedded_ha
+        base_url, session_id, _config, _container = embedded_ha
         resp = _mcp_post(
             base_url,
             {
@@ -365,6 +401,40 @@ def test_read_only_tool_call(self, embedded_ha):
         # The tool ran against the real HA instance and returned content.
         assert parsed["result"].get("content"), parsed
 
+    def test_invalidate_caches_recovers_from_editable_finder_keyerror(
+        self, embedded_ha
+    ):
+        """_safe_invalidate_caches() recovers from the 3.14 editable-finder
+        KeyError on the REAL component code + container interpreter (#1891/#1985).
+
+        The reaching-ready ``embedded_ha`` fixture already proves the happy path
+        (bring-up runs ``_installed_ha_mcp_version`` → ``_safe_invalidate_caches``
+        without crashing). This drives the FAILURE path: a ``python3`` process in
+        the same container injects the exact KeyError and asserts the helper
+        prunes the stale entry and the retry completes. Reverting the fix makes
+        the probe exit non-zero.
+
+        NOTE (skip-ceiling coupling): like its siblings this module is
+        ``container_only`` + ``not_on_embedded`` (see the file-level
+        ``pytestmark``), so this test is SKIPPED on the haos, haos_inaddon,
+        haos_embedded, and embedded lanes and RUNS only on the container lane.
+        Each such skip counts toward ``_SKIP_CEILING_PER_LANE`` in
+        tests/src/e2e/basic/test_backend_dispatch_smoke.py — adding this test
+        tripped ``test_session_skipped_count_below_ceiling`` until those four
+        ceilings were bumped by 1. Any future marker-gated test added here will
+        trip that guard the same way until its ceilings are bumped.
+        """
+        _base_url, _session_id, _config, container = embedded_ha
+        result = container.get_wrapped_container().exec_run(
+            ["python3", "-c", _INVALIDATE_RECOVERY_PROBE]
+        )
+        output = (result.output or b"").decode("utf-8", "replace")
+        assert result.exit_code == 0, (
+            "in-container _safe_invalidate_caches() did not recover from the "
+            f"injected editable-finder KeyError (exit {result.exit_code}):\n{output}"
+        )
+        assert "RECOVERY_OK" in output, output
+
     def test_llm_api_registered_inside_ha(self, embedded_ha):
         """The bring-up registered the toolset as an LLM API in the REAL HA.
 
@@ -376,7 +446,7 @@ def test_llm_api_registered_inside_ha(self, embedded_ha):
         (registration runs right after webhook bring-up and imports the mcp
         SDK on the executor first), so the log is polled briefly.
         """
-        _base_url, _session_id, config_path = embedded_ha
+        _base_url, _session_id, config_path, _container = embedded_ha
         log_file = config_path / "home-assistant.log"
         needle = "Registered the HA-MCP toolset as LLM API"
         deadline = time.monotonic() + 60
@@ -413,7 +483,7 @@ async def test_llm_api_client_path_full_catalog(self, embedded_ha):
         from mcp.client.streamable_http import streamable_http_client
         from voluptuous_openapi import convert_to_voluptuous
 
-        base_url, _session_id, _config = embedded_ha
+        base_url, _session_id, _config, _container = embedded_ha
         url = f"{base_url}/api/webhook/{_WEBHOOK_ID}"
 
         async with asyncio.timeout(120):

tests/src/unit/test_embedded_server.py

@@ -1402,6 +1402,110 @@ def _version(name):
         assert es._installed_dist_version(DIST_NAME_STABLE) is None
 
 
+class TestSafeInvalidateCaches:
+    """Guard for the Python-3.14 setuptools editable-finder KeyError (#1891, #1985).
+
+    On Python 3.14, ``homeassistant`` as a setuptools *editable* install makes
+    ``PathFinder.invalidate_caches()`` raise ``KeyError`` at its
+    ``del sys.path_importer_cache[name]`` line, aborting
+    ``importlib.invalidate_caches()`` and crashing in-process bring-up. The
+    helper prunes the stale entries CPython choked on, then re-runs the call so
+    the full sweep (finder invalidation + namespace-path epoch + metadata)
+    completes.
+    """
+
+    _EDITABLE_KEY = "__editable__.homeassistant-2026.7.2.finder.__path_hook__"
+
+    @pytest.fixture(autouse=True)
+    def _isolate_path_cache(self, monkeypatch):
+        # Recovery prunes sys.path_importer_cache entries; hand each test a
+        # private copy so it never mutates the real process cache.
+        monkeypatch.setattr(sys, "path_importer_cache", dict(sys.path_importer_cache))
+
+    def _raise_once(self, monkeypatch, *, then=None):
+        """Patch invalidate_caches to raise the editable KeyError on the first
+        call, then delegate to ``then`` (a no-op by default) on the retry."""
+        calls = []
+
+        def _invalidate():
+            calls.append(1)
+            if len(calls) == 1:
+                raise KeyError(self._EDITABLE_KEY)
+            if then is not None:
+                then()
+
+        monkeypatch.setattr(es.importlib, "invalidate_caches", _invalidate)
+        return calls
+
+    def test_no_recovery_when_first_call_succeeds(self, monkeypatch):
+        calls = []
+        monkeypatch.setattr(es.importlib, "invalidate_caches", lambda: calls.append(1))
+        es._safe_invalidate_caches()
+        assert calls == [1]  # succeeded first try, no retry
+
+    def test_prunes_stale_entries_then_retries(self, monkeypatch):
+        calls = self._raise_once(monkeypatch)
+        sys.path_importer_cache.clear()
+        sys.path_importer_cache["/abs/deps/dir"] = object()  # abs + live → kept
+        sys.path_importer_cache[self._EDITABLE_KEY] = None  # non-abs None → pruned
+        sys.path_importer_cache[""] = None  # relative None → pruned
+        # Non-absolute key with a LIVE (non-None) finder: the None disjunct is
+        # False here, so ONLY the ``not os.path.isabs`` disjunct can prune it —
+        # this independently exercises the abspath branch (a real editable
+        # placeholder has a live finder, not None).
+        sys.path_importer_cache["rel/not/absolute"] = object()
+
+        es._safe_invalidate_caches()
+
+        assert len(calls) == 2  # aborted once, retried once
+        assert self._EDITABLE_KEY not in sys.path_importer_cache
+        assert "" not in sys.path_importer_cache
+        assert "rel/not/absolute" not in sys.path_importer_cache
+        assert "/abs/deps/dir" in sys.path_importer_cache
+
+    def test_propagates_non_keyerror(self, monkeypatch):
+        def _boom():
+            raise RuntimeError("unrelated import-system failure")
+
+        monkeypatch.setattr(es.importlib, "invalidate_caches", _boom)
+        with pytest.raises(RuntimeError):
+            es._safe_invalidate_caches()
+
+    def test_retry_keyerror_is_tolerated(self, monkeypatch):
+        # If a concurrent import re-adds a stale placeholder between the prune and
+        # the retry, the retry can raise KeyError a SECOND time. That must not
+        # re-crash bring-up (the helper's whole purpose) — it is logged
+        # best-effort and swallowed, not re-raised.
+        def _always_keyerror():
+            raise KeyError(self._EDITABLE_KEY)
+
+        monkeypatch.setattr(es.importlib, "invalidate_caches", _always_keyerror)
+        es._safe_invalidate_caches()  # must not raise despite the retry also failing
+
+    def test_recovery_advances_namespace_epoch(self, monkeypatch):
+        # Codex (#1987): recovery must preserve PathFinder's namespace-path epoch
+        # bump so a newly-installed PEP 420 namespace portion stays discoverable.
+        # The retry runs the REAL invalidate_caches, which advances the epoch.
+        from importlib._bootstrap_external import _NamespacePath
+
+        real = importlib.invalidate_caches
+        self._raise_once(monkeypatch, then=real)
+
+        before = _NamespacePath._epoch
+        es._safe_invalidate_caches()
+        assert _NamespacePath._epoch > before
+
+    def test_installed_version_survives_editable_finder_keyerror(self, monkeypatch):
+        # End-to-end regression for the exact reported traceback:
+        # _installed_ha_mcp_version() -> importlib.invalidate_caches() raised
+        # KeyError and crashed bring-up. With the guard the version still
+        # resolves.
+        self._raise_once(monkeypatch)
+        monkeypatch.setattr(es.importlib.util, "find_spec", lambda name: object())
+        monkeypatch.setattr(importlib.metadata, "version", lambda name: "7.14.1")
+        assert es._installed_ha_mcp_version() == "7.14.1"
+
+
 # ---------------------------------------------------------------------------
 # Worker-thread env staging
 # ---------------------------------------------------------------------------