fix: parallelize deep_search Tier 3 config fetches (closes #879)

src/ha_mcp/tools/smart_search.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import logging
+import os
 import random
 import time
 from typing import Any
@@ -23,11 +24,17 @@
 BULK_WEBSOCKET_TIMEOUT = 3.0  # Timeout for bulk WebSocket calls
 INDIVIDUAL_CONFIG_TIMEOUT = 5.0  # Timeout for individual config fetches
 
-# Time budgets for fallback individual fetching (in seconds)
-AUTOMATION_CONFIG_TIME_BUDGET = (
-    15.0  # Max time for fetching automation configs individually
+# Time budgets for fallback individual fetching (in seconds).
+# Configurable via env vars for instances with many automations/scripts.
+AUTOMATION_CONFIG_TIME_BUDGET = float(
+    os.environ.get("HAMCP_AUTOMATION_CONFIG_TIME_BUDGET", "30")
 )
-SCRIPT_CONFIG_TIME_BUDGET = 10.0  # Max time for fetching script configs individually
+SCRIPT_CONFIG_TIME_BUDGET = float(
+    os.environ.get("HAMCP_SCRIPT_CONFIG_TIME_BUDGET", "20")
+)
+
+# Batch size for parallel individual config fetches (Tier 3 fallback)
+INDIVIDUAL_FETCH_BATCH_SIZE = 10
 
 
 def _simplify_states_summary(
@@ -903,39 +910,59 @@ async def deep_search(
                                 f"Automation WebSocket bulk fetch ({ws_type}) failed: {e}"
                             )
 
-                # Attempt C: Individual REST calls with time budget (LAST RESORT)
-                # Prioritize name-matched automations so we at least get their configs
+                # Attempt C: Parallel individual REST calls with time budget (LAST RESORT)
+                # Fetch ALL configs in parallel batches — don't prioritize by name score.
+                # Name score is only used for result ranking, not fetch order, because
+                # deep_search's purpose is to find matches INSIDE configs (conditions/actions),
+                # not just by name. Prioritizing by name would skip the configs most likely
+                # to contain non-obvious matches. See #879.
                 if not bulk_fetched:
                     budget_start = time.perf_counter()
-                    sorted_by_score = sorted(
-                        name_scored, key=lambda x: x[2], reverse=True
-                    )
+                    uids_to_fetch = [
+                        uid
+                        for _, _, _, uid in name_scored
+                        if uid and uid not in all_automation_configs
+                    ]
+                    total_to_fetch = len(uids_to_fetch)
+                    fetched_count = 0
 
-                    for (
-                        _entity_id,
-                        _friendly_name,
-                        _name_score,
-                        unique_id,
-                    ) in sorted_by_score:
-                        if (
-                            time.perf_counter() - budget_start
-                            > AUTOMATION_CONFIG_TIME_BUDGET
-                        ):
-                            break
-                        if not unique_id or unique_id in all_automation_configs:
-                            continue
+                    async def _fetch_automation_config(uid: str) -> tuple[str, dict[str, Any] | None]:
                         try:
                             config = await asyncio.wait_for(
                                 self.client._request(
-                                    "GET", f"/config/automation/config/{unique_id}"
+                                    "GET", f"/config/automation/config/{uid}"
                                 ),
                                 timeout=INDIVIDUAL_CONFIG_TIMEOUT,
                             )
-                            all_automation_configs[unique_id] = config
+                            return (uid, config)
                         except Exception as e:
                             logger.debug(
-                                f"Automation individual config fetch ({unique_id}) failed: {e}"
+                                f"Automation individual config fetch ({uid}) failed: {e}"
                             )
+                            return (uid, None)
+
+                    for i in range(0, len(uids_to_fetch), INDIVIDUAL_FETCH_BATCH_SIZE):
+                        if (
+                            time.perf_counter() - budget_start
+                            > AUTOMATION_CONFIG_TIME_BUDGET
+                        ):
+                            skipped = total_to_fetch - fetched_count
+                            logger.warning(
+                                f"Automation config fetch budget exhausted "
+                                f"({AUTOMATION_CONFIG_TIME_BUDGET}s). "
+                                f"Fetched {fetched_count}/{total_to_fetch}, "
+                                f"skipped {skipped} automations."
+                            )
+                            break
+                        batch = uids_to_fetch[i : i + INDIVIDUAL_FETCH_BATCH_SIZE]
+                        results = await asyncio.gather(
+                            *[_fetch_automation_config(uid) for uid in batch],
+                            return_exceptions=True,
+                        )
+                        for result in results:
+                            if isinstance(result, tuple) and result[1] is not None:
+                                all_automation_configs[result[0]] = result[1]
+                                fetched_count += 1
 
                 # Phase 3: Score with whatever configs we have
                 for entity_id, friendly_name, name_score, unique_id in name_scored:
@@ -1040,37 +1067,52 @@ async def deep_search(
                                 f"Script WebSocket bulk fetch ({ws_type}) failed: {e}"
                             )
 
-                # Attempt C: Individual fetch with budget
+                # Attempt C: Parallel individual fetch with budget (see #879)
                 if not script_bulk_fetched:
                     budget_start = time.perf_counter()
-                    sorted_scripts = sorted(
-                        script_name_scored, key=lambda x: x[3], reverse=True
-                    )
-                    for (
-                        _entity_id,
-                        _friendly_name,
-                        script_id,
-                        _name_score,
-                    ) in sorted_scripts:
-                        if (
-                            time.perf_counter() - budget_start
-                            > SCRIPT_CONFIG_TIME_BUDGET
-                        ):
-                            break
-                        if script_id in all_script_configs:
-                            continue
+                    sids_to_fetch = [
+                        sid
+                        for _, _, sid, _ in script_name_scored
+                        if sid and sid not in all_script_configs
+                    ]
+                    total_to_fetch = len(sids_to_fetch)
+                    fetched_count = 0
+
+                    async def _fetch_script_config(sid: str) -> tuple[str, dict[str, Any] | None]:
                         try:
                             config_resp = await asyncio.wait_for(
-                                self.client.get_script_config(script_id),
+                                self.client.get_script_config(sid),
                                 timeout=INDIVIDUAL_CONFIG_TIMEOUT,
                             )
-                            all_script_configs[script_id] = config_resp.get(
-                                "config", {}
-                            )
+                            return (sid, config_resp.get("config", {}))
                         except Exception as e:
                             logger.debug(
-                                f"Script individual config fetch ({script_id}) failed: {e}"
+                                f"Script individual config fetch ({sid}) failed: {e}"
                             )
+                            return (sid, None)
+
+                    for i in range(0, len(sids_to_fetch), INDIVIDUAL_FETCH_BATCH_SIZE):
+                        if (
+                            time.perf_counter() - budget_start
+                            > SCRIPT_CONFIG_TIME_BUDGET
+                        ):
+                            skipped = total_to_fetch - fetched_count
+                            logger.warning(
+                                f"Script config fetch budget exhausted "
+                                f"({SCRIPT_CONFIG_TIME_BUDGET}s). "
+                                f"Fetched {fetched_count}/{total_to_fetch}, "
+                                f"skipped {skipped} scripts."
+                            )
+                            break
+                        batch = sids_to_fetch[i : i + INDIVIDUAL_FETCH_BATCH_SIZE]
+                        results = await asyncio.gather(
+                            *[_fetch_script_config(sid) for sid in batch],
+                            return_exceptions=True,
+                        )
+                        for result in results:
+                            if isinstance(result, tuple) and result[1] is not None:
+                                all_script_configs[result[0]] = result[1]
+                                fetched_count += 1
 
                 # Phase 3: Score scripts
                 for (

tests/src/unit/test_deep_search_tier3_parallel.py

@@ -0,0 +1,187 @@
+"""Unit tests for parallel Tier 3 config fetching in deep_search.
+
+Validates that when bulk config fetches fail (Tier 1 & 2), Tier 3 fetches
+configs in parallel batches without name-score prioritization. This ensures
+entities referenced only inside automation conditions/actions (not in the
+automation name) are still found. Regression test for #879.
+"""
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from ha_mcp.tools.smart_search import SmartSearchTools
+
+
+def _make_entity(entity_id: str, friendly_name: str) -> dict:
+    return {
+        "entity_id": entity_id,
+        "state": "on",
+        "attributes": {"friendly_name": friendly_name},
+    }
+
+
+def _make_automation_entities(count: int) -> list[dict]:
+    """Create automation entities with unique IDs in attributes."""
+    return [
+        {
+            "entity_id": f"automation.auto_{i}",
+            "state": "on",
+            "attributes": {
+                "friendly_name": f"Automation {i}",
+                "id": f"uid_{i}",
+            },
+        }
+        for i in range(count)
+    ]
+
+
+class TestTier3ParallelFetch:
+    """Test that Tier 3 fetches configs in parallel without name-score prioritization."""
+
+    @pytest.fixture
+    def mock_client(self):
+        client = MagicMock()
+        client.get_config = AsyncMock(return_value={"time_zone": "UTC"})
+        # Bulk fetch fails (triggers Tier 3)
+        client._request = AsyncMock(side_effect=Exception("Bulk fetch unavailable"))
+        client.send_websocket_message = AsyncMock(
+            side_effect=Exception("WebSocket unavailable")
+        )
+        return client
+
+    @pytest.fixture
+    def smart_tools(self, mock_client):
+        return SmartSearchTools(mock_client)
+
+    @pytest.mark.asyncio
+    async def test_tier3_fetches_all_configs_not_just_name_matches(
+        self, mock_client, smart_tools
+    ):
+        """Configs for ALL automations should be fetched, not just name-matched ones.
+
+        Regression test for #879: an automation named "Morning Routine" that
+        references "sensor.kitchen_temp" in its conditions should be found when
+        searching for "kitchen_temp", even though the name doesn't match.
+        """
+        # Create automations: only auto_2 has the search term in its config,
+        # but its NAME doesn't match the query at all.
+        automations = [
+            {
+                "entity_id": "automation.morning_routine",
+                "state": "on",
+                "attributes": {
+                    "friendly_name": "Morning Routine",
+                    "id": "uid_morning",
+                },
+            },
+            {
+                "entity_id": "automation.evening_lights",
+                "state": "on",
+                "attributes": {
+                    "friendly_name": "Evening Lights",
+                    "id": "uid_evening",
+                },
+            },
+        ]
+
+        all_entities = automations + [
+            _make_entity("sensor.kitchen_temp", "Kitchen Temperature"),
+        ]
+
+        mock_client.get_states = AsyncMock(return_value=all_entities)
+
+        # Track which UIDs get fetched
+        fetched_uids = []
+
+        async def _individual_fetch(method: str, url: str) -> dict:
+            uid = url.split("/")[-1]
+            fetched_uids.append(uid)
+            # "Morning Routine" references sensor.kitchen_temp in condition
+            if uid == "uid_morning":
+                return {
+                    "id": uid,
+                    "trigger": [{"platform": "time", "at": "07:00"}],
+                    "condition": [
+                        {
+                            "condition": "numeric_state",
+                            "entity_id": "sensor.kitchen_temp",
+                            "below": 20,
+                        }
+                    ],
+                    "action": [{"service": "light.turn_on"}],
+                }
+            return {
+                "id": uid,
+                "trigger": [{"platform": "time", "at": "18:00"}],
+                "action": [{"service": "light.turn_on", "target": {"entity_id": "light.living_room"}}],
+            }
+
+        mock_client._request = AsyncMock(side_effect=_individual_fetch)
+        # Keep WebSocket failing to force Tier 3
+        mock_client.send_websocket_message = AsyncMock(
+            side_effect=Exception("WebSocket unavailable")
+        )
+
+        result = await smart_tools.deep_search(
+            query="kitchen_temp",
+            search_types=["automation"],
+            limit=10,
+        )
+
+        # Both UIDs should have been fetched (not just name-matched ones)
+        assert "uid_morning" in fetched_uids, (
+            "Morning Routine config should be fetched even though "
+            "its name doesn't match 'kitchen_temp'"
+        )
+        assert "uid_evening" in fetched_uids, (
+            "All automation configs should be fetched in Tier 3"
+        )
+
+        # The search should find the automation that references kitchen_temp
+        auto_results = result.get("automations", [])
+        matched_ids = [r["entity_id"] for r in auto_results]
+        assert "automation.morning_routine" in matched_ids, (
+            f"Should find automation referencing kitchen_temp in conditions. "
+            f"Got: {matched_ids}"
+        )
+
+    @pytest.mark.asyncio
+    async def test_tier3_respects_time_budget(self, mock_client, smart_tools):
+        """Tier 3 should stop fetching when time budget is exhausted."""
+        automations = _make_automation_entities(30)
+        mock_client.get_states = AsyncMock(return_value=automations)
+
+        call_count = 0
+
+        async def _slow_fetch(method: str, url: str) -> dict:
+            nonlocal call_count
+            uid = url.split("/")[-1]
+            # First call triggers bulk fetch failure
+            if url.rstrip("/") == "/config/automation/config":
+                raise Exception("Bulk unavailable")
+            call_count += 1
+            await asyncio.sleep(0.5)  # Simulate slow fetches
+            return {"id": uid, "action": []}
+
+        mock_client._request = AsyncMock(side_effect=_slow_fetch)
+        mock_client.send_websocket_message = AsyncMock(
+            side_effect=Exception("WebSocket unavailable")
+        )
+
+        with patch(
+            "ha_mcp.tools.smart_search.AUTOMATION_CONFIG_TIME_BUDGET", 2.0
+        ):
+            result = await smart_tools.deep_search(
+                query="test",
+                search_types=["automation"],
+                limit=10,
+            )
+
+        # With 2s budget and 0.5s per fetch in batches of 10,
+        # we should fetch some but not all 30
+        assert call_count < 30, (
+            f"Should stop before fetching all 30, but fetched {call_count}"
+        )
+        assert call_count > 0, "Should fetch at least one batch"