fix(chroma): read list_drawers and tunnels from sqlite, skip HNSW

mempalace/backends/chroma.py

@@ -1203,6 +1203,148 @@ def _sqlite_wing_room_counts(
     return total, wing_rooms
 
 
+def sqlite_room_wing_hall_counts(palace_path: str, collection_name: str) -> Optional[list[tuple]]:
+    """Grouped (room, wing, hall, n) from ``chroma.sqlite3``, or ``None``."""
+    db_path = os.path.join(palace_path, "chroma.sqlite3")
+    if not os.path.isfile(db_path):
+        return None
+    try:
+        conn = sqlite3.connect(sqlite_read_uri(db_path), uri=True)
+        try:
+            conn.execute("PRAGMA busy_timeout = 3000")
+            if (
+                conn.execute(
+                    "SELECT 1 FROM collections WHERE name = ?", (collection_name,)
+                ).fetchone()
+                is None
+            ):
+                return None
+            return conn.execute(
+                """
+                SELECT
+                    COALESCE(rm.string_value, CAST(rm.int_value AS TEXT),
+                             CAST(rm.float_value AS TEXT), '') AS room,
+                    COALESCE(wm.string_value, CAST(wm.int_value AS TEXT),
+                             CAST(wm.float_value AS TEXT), '') AS wing,
+                    COALESCE(hm.string_value, CAST(hm.int_value AS TEXT),
+                             CAST(hm.float_value AS TEXT), '') AS hall,
+                    COUNT(*) AS n
+                FROM embeddings e
+                JOIN segments s ON e.segment_id = s.id AND s.scope = 'METADATA'
+                JOIN collections c ON s.collection = c.id
+                LEFT JOIN embedding_metadata rm ON rm.id = e.id AND rm.key = 'room'
+                LEFT JOIN embedding_metadata wm ON wm.id = e.id AND wm.key = 'wing'
+                LEFT JOIN embedding_metadata hm ON hm.id = e.id AND hm.key = 'hall'
+                WHERE c.name = ?
+                GROUP BY room, wing, hall
+                """,
+                (collection_name,),
+            ).fetchall()
+        finally:
+            conn.close()
+    except sqlite3.Error:
+        return None
+
+
+def sqlite_list_id_metadata(
+    palace_path: str,
+    collection_name: str,
+    where: Optional[dict] = None,
+) -> Optional[tuple[list[str], list[str], list[dict]]]:
+    """All drawer ids + metadata from sqlite, without opening HNSW.
+
+    ``where`` supports equality on ``wing``/``room`` and ``$and`` of those,
+    matching ``tool_list_drawers``. Returns ``None`` when sqlite cannot be
+    trusted so the caller can fall back to ``col.get`` paging.
+    """
+    db_path = os.path.join(palace_path, "chroma.sqlite3")
+    if not os.path.isfile(db_path):
+        return None
+    wing_eq = room_eq = None
+    if where:
+        if not isinstance(where, dict):
+            return None
+        if list(where.keys()) == ["$and"]:
+            for child in where["$and"] or []:
+                if not isinstance(child, dict) or len(child) != 1:
+                    return None
+                key, val = next(iter(child.items()))
+                if key == "wing":
+                    wing_eq = val
+                elif key == "room":
+                    room_eq = val
+                else:
+                    return None
+        elif len(where) == 1 and not next(iter(where.keys())).startswith("$"):
+            key, val = next(iter(where.items()))
+            if key == "wing":
+                wing_eq = val
+            elif key == "room":
+                room_eq = val
+            else:
+                return None
+        else:
+            return None
+    try:
+        conn = sqlite3.connect(sqlite_read_uri(db_path), uri=True)
+        try:
+            conn.execute("PRAGMA busy_timeout = 3000")
+            if (
+                conn.execute(
+                    "SELECT 1 FROM collections WHERE name = ?", (collection_name,)
+                ).fetchone()
+                is None
+            ):
+                return None
+            rows = conn.execute(
+                """
+                SELECT e.embedding_id, m.key, m.string_value, m.int_value, m.float_value
+                FROM embeddings e
+                JOIN segments s ON e.segment_id = s.id AND s.scope = 'METADATA'
+                JOIN collections c ON s.collection = c.id
+                LEFT JOIN embedding_metadata m ON m.id = e.id
+                WHERE c.name = ?
+                ORDER BY e.id
+                """,
+                (collection_name,),
+            ).fetchall()
+        finally:
+            conn.close()
+    except sqlite3.Error:
+        return None
+
+    by_id: dict[str, dict] = {}
+    docs_by_id: dict[str, str] = {}
+    order: list[str] = []
+    for embedding_id, key, sval, ival, fval in rows:
+        doc_id = str(embedding_id)
+        if doc_id not in by_id:
+            by_id[doc_id] = {}
+            order.append(doc_id)
+        if not key:
+            continue
+        value = sval if sval is not None else (ival if ival is not None else fval)
+        if value is None:
+            continue
+        if str(key) == "chroma:document":
+            docs_by_id[doc_id] = str(value)
+        else:
+            by_id[doc_id][str(key)] = value
+    ids: list[str] = []
+    metas: list[dict] = []
+    documents: list[str] = []
+    for doc_id in order:
+        meta = by_id[doc_id]
+        if wing_eq is not None and meta.get("wing") != wing_eq:
+            continue
+        if room_eq is not None and meta.get("room") != room_eq:
+            continue
+        ids.append(doc_id)
+        metas.append(meta)
+        documents.append(docs_by_id.get(doc_id, ""))
+    return ids, documents, metas
+
+
 def _pin_hnsw_threads(collection) -> None:
     """Best-effort retrofit: pin ``hnsw:num_threads=1`` on an existing collection.
 

mempalace/mcp_server.py

@@ -2047,45 +2047,11 @@ def _graph_stats_from_grouped_rows(rows):
 
 
 def _chroma_room_wing_hall_counts():
-    import sqlite3 as _sqlite3
-
     if not _config.palace_path:
         return None
-    db_path = os.path.join(_config.palace_path, "chroma.sqlite3")
-    if not os.path.isfile(db_path):
-        return None
-    collection_name = _config.collection_name
-    conn = _sqlite3.connect(sqlite_read_uri(db_path), uri=True)
-    try:
-        conn.execute("PRAGMA busy_timeout = 3000")
-        if (
-            conn.execute("SELECT 1 FROM collections WHERE name = ?", (collection_name,)).fetchone()
-            is None
-        ):
-            return None
-        return conn.execute(
-            """
-            SELECT
-                COALESCE(rm.string_value, CAST(rm.int_value AS TEXT),
-                         CAST(rm.float_value AS TEXT), '') AS room,
-                COALESCE(wm.string_value, CAST(wm.int_value AS TEXT),
-                         CAST(wm.float_value AS TEXT), '') AS wing,
-                COALESCE(hm.string_value, CAST(hm.int_value AS TEXT),
-                         CAST(hm.float_value AS TEXT), '') AS hall,
-                COUNT(*) AS n
-            FROM embeddings e
-            JOIN segments s ON e.segment_id = s.id AND s.scope = 'METADATA'
-            JOIN collections c ON s.collection = c.id
-            LEFT JOIN embedding_metadata rm ON rm.id = e.id AND rm.key = 'room'
-            LEFT JOIN embedding_metadata wm ON wm.id = e.id AND wm.key = 'wing'
-            LEFT JOIN embedding_metadata hm ON hm.id = e.id AND hm.key = 'hall'
-            WHERE c.name = ?
-            GROUP BY room, wing, hall
-            """,
-            (collection_name,),
-        ).fetchall()
-    finally:
-        conn.close()
+    from .backends.chroma import sqlite_room_wing_hall_counts
+
+    return sqlite_room_wing_hall_counts(_config.palace_path, _config.collection_name)
 
 
 def tool_status():
@@ -2530,10 +2496,8 @@ def tool_get_aaak_spec():
 def tool_traverse_graph(start_room: str, max_hops: int = 2):
     """Walk the palace graph from a room. Find connected ideas across wings."""
     max_hops = max(1, min(max_hops, 10))
-    col = _get_collection()
-    if not col:
-        return _collection_error_or_no_palace()
-    return traverse(start_room, col=col, max_hops=max_hops)
+    # sqlite metadata path does not open HNSW; build_graph falls back itself.
+    return traverse(start_room, max_hops=max_hops, config=_config)
 
 
 def tool_find_tunnels(wing_a: str = None, wing_b: str = None):
@@ -2543,10 +2507,7 @@ def tool_find_tunnels(wing_a: str = None, wing_b: str = None):
         wing_b = _sanitize_optional_name(wing_b, "wing_b")
     except ValueError as e:
         return {"error": str(e)}
-    col = _get_collection()
-    if not col:
-        return _collection_error_or_no_palace()
-    return find_tunnels(wing_a, wing_b, col=col)
+    return find_tunnels(wing_a, wing_b, config=_config)
 
 
 def tool_graph_stats():
@@ -3625,10 +3586,6 @@ def tool_list_drawers(
     except ValueError as e:
         return {"error": str(e)}
 
-    col = _get_collection()
-    if not col:
-        return _collection_error_or_no_palace()
-
     try:
         where = None
         conditions = []
@@ -3643,7 +3600,21 @@ def tool_list_drawers(
         elif len(conditions) > 1:
             where = {"$and": conditions}
 
-        ids, documents, metadatas = _fetch_drawer_rows(col, where=where, include=["metadatas"])
+        ids = documents = metadatas = None
+        listed = None
+        if _is_chroma_backend() and _config.palace_path:
+            from .backends.chroma import sqlite_list_id_metadata
+
+            listed = sqlite_list_id_metadata(
+                _config.palace_path, _config.collection_name, where=where
+            )
+        if listed is not None:
+            ids, documents, metadatas = listed
+        else:
+            col = _get_collection()
+            if not col:
+                return _collection_error_or_no_palace()
+            ids, documents, metadatas = _fetch_drawer_rows(col, where=where, include=["metadatas"])
         drawers = _collapse_drawer_rows(ids, documents, metadatas)
 
         if since_dt is not None or before_dt is not None:
@@ -3654,7 +3625,10 @@ def tool_list_drawers(
             ]
 
         page = drawers[offset : offset + limit]
-        _fill_drawer_previews(col, page)
+        if listed is None:
+            col = _get_collection()
+            if col:
+                _fill_drawer_previews(col, page)
 
         return {
             "drawers": page,

mempalace/palace_graph.py

@@ -66,6 +66,76 @@ def _normalize_wing(wing: str | None) -> str | None:
 _GRAPH_CACHE_TTL = 60.0  # seconds — graph changes less often than metadata
 
 
+def _try_sqlite_nodes_edges(config=None):
+    """Build graph nodes/edges from backend sqlite metadata, no HNSW.
+
+    Returns ``(nodes, edges)`` or ``None`` when the palace is not a sqlite
+    backend we know how to read. Dates are omitted (empty lists) — tunnel
+    ``recent`` stays blank on this path.
+    """
+    config = config or MempalaceConfig()
+    palace = config.palace_path
+    name = config.collection_name
+    if not palace:
+        return None
+    rows = None
+    chroma_db = os.path.join(palace, "chroma.sqlite3")
+    exact_db = os.path.join(palace, "sqlite_exact.sqlite3")
+    try:
+        if os.path.isfile(chroma_db):
+            from .backends.chroma import sqlite_room_wing_hall_counts
+
+            rows = sqlite_room_wing_hall_counts(palace, name)
+        elif os.path.isfile(exact_db):
+            from .backends.sqlite_exact import sqlite_room_wing_hall_counts
+
+            rows = sqlite_room_wing_hall_counts(palace, name)
+    except Exception:
+        logger.debug("sqlite graph path failed; falling back to client paging", exc_info=True)
+        return None
+    if rows is None:
+        return None
+    return _nodes_edges_from_grouped_rows(rows)
+
+
+def _nodes_edges_from_grouped_rows(rows):
+    """Mirror ``build_graph``'s per-drawer filter from ``(room, wing, hall, n)``."""
+    room_data = defaultdict(lambda: {"wings": set(), "halls": set(), "count": 0})
+    for room, wing, hall, n in rows:
+        if not room or room == "general" or not wing:
+            continue
+        node = room_data[str(room)]
+        node["wings"].add(str(wing))
+        if hall:
+            node["halls"].add(str(hall))
+        node["count"] += int(n)
+    edges = []
+    nodes = {}
+    for room, data in room_data.items():
+        wings = sorted(data["wings"])
+        halls = sorted(data["halls"])
+        nodes[room] = {
+            "wings": wings,
+            "halls": halls,
+            "count": data["count"],
+            "dates": [],
+        }
+        if len(wings) >= 2:
+            for i, wa in enumerate(wings):
+                for wb in wings[i + 1 :]:
+                    for hall in halls:
+                        edges.append(
+                            {
+                                "room": room,
+                                "wing_a": wa,
+                                "wing_b": wb,
+                                "hall": hall,
+                                "count": data["count"],
+                            }
+                        )
+    return nodes, edges
+
+
 def invalidate_graph_cache():
     """Clear the graph cache. Called from mcp_server.py on writes."""
     global _graph_cache_nodes, _graph_cache_edges, _graph_cache_time
@@ -110,7 +180,18 @@ def build_graph(col=None, config=None):
         if _graph_cache_nodes is not None and (now - _graph_cache_time) < _GRAPH_CACHE_TTL:
             return _graph_cache_nodes, _graph_cache_edges
 
+    # Only when the caller did not pass a collection: MCP tools. Tests that
+    # inject ``col=`` keep the client paging path against that collection.
     if col is None:
+        sqlite_graph = _try_sqlite_nodes_edges(config)
+        if sqlite_graph is not None:
+            nodes, edges = sqlite_graph
+            if nodes:
+                with _graph_cache_lock:
+                    _graph_cache_nodes = nodes
+                    _graph_cache_edges = edges
+                    _graph_cache_time = time.time()
+            return nodes, edges
         col = _get_collection(config)
     if not col:
         return {}, []