fix(status): wire CLI status() to get_all_metadata() fast path (#2153)

CLI `mempalace status` (`miner.py:status()`) still used an O(n²)
offset-paginated loop even after the MCP server path was fixed in
#1796/#1836. For Qdrant, each `col.get(offset=N)` call materializes the
entire collection via `_scroll_all()` then slices — so ~32 full re-fetches
for a 159k-drawer palace.

Fix: after the chroma sqlite fast path returns None and the collection is
opened, check `getattr(col, "get_all_metadata", None)`. If callable,
fetch all metadata in one pass, tally wing/room counts, and return. The
existing offset loop remains as fallback for backends that don't expose
the contract method.

This mirrors the delegation pattern already used by
`mcp_server._fetch_all_metadata()` (#1796).

Before: 1m46s on 159,601-drawer Qdrant palace
After:   3.4s (31x speedup)

Test: `tests/test_cli_status_fast_path.py` asserts status() uses
get_all_metadata() when available and does NOT call the offset loop,
plus fallback coverage for legacy collections.

Author: javapacr

mempalace/miner.py

@@ -2161,6 +2161,18 @@ def status(palace_path: str):
         print("  Run `mempalace repair --mode from-sqlite --archive-existing` first.")
         return
 
+    # Fast path: single-pass metadata fetch for backends that expose
+    # get_all_metadata() (#1796). Avoids the O(n^2) offset loop below.
+    get_all = getattr(col, "get_all_metadata", None)
+    if callable(get_all):
+        metas = get_all()
+        wing_rooms: dict = defaultdict(lambda: defaultdict(int))
+        for m in metas:
+            m = m or {}
+            wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1
+        _print_status(len(metas), wing_rooms)
+        return
+
     # Count by wing and room — paginate to avoid SQLite "too many SQL
     # variables" error on large palaces (see #802, #850).
     total = col.count()

tests/test_cli_status_fast_path.py

@@ -0,0 +1,244 @@
+"""
+Tests for issue #2153 -- CLI ``mempalace status`` O(n^2) offset loop on
+backends that expose ``get_all_metadata()``.
+
+The CLI path (``miner.status()``) was never wired to the
+``get_all_metadata()`` contract method that the MCP server already uses
+(``mcp_server._fetch_all_metadata``, PR #1796).  For backends like Qdrant
+whose ``get(limit=, offset=)`` is backed by a full ``_scroll_all()``
+materialization, the offset loop re-walks the entire collection on every
+page -- O(n^2) in collection size.
+
+These tests verify the fix: ``miner.status()`` must use
+``get_all_metadata()`` when the collection exposes it, and must NOT fall
+through to the offset-paginated ``col.get()`` loop.  The fallback path is
+also exercised for collections that predate the contract method.
+"""
+
+from unittest import mock
+
+import pytest
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_collection_with_get_all(metadata_list):
+    """A mock collection that exposes get_all_metadata() and tracks get() calls.
+
+    The returned object records every call to ``get()`` so tests can assert
+    the offset loop was NOT entered.
+    """
+
+    class _Collection:
+        def __init__(self):
+            self._meta = metadata_list
+            self.get_calls = []
+
+        def get_all_metadata(self, where=None):
+            return list(self._meta)
+
+        def get(self, *, ids=None, where=None, limit=None, offset=None, include=None):
+            self.get_calls.append({"limit": limit, "offset": offset})
+            # Simulate what a Qdrant backend does: materialize everything, slice
+            offset = offset or 0
+            limit = limit if limit is not None else len(self._meta)
+            page = self._meta[offset : offset + limit]
+            return {"ids": [], "documents": [], "metadatas": page}
+
+        def count(self):
+            return len(self._meta)
+
+    return _Collection()
+
+
+class _LegacyCollection:
+    """Collection WITHOUT get_all_metadata() -- triggers the fallback loop."""
+
+    def __init__(self, metadata_list):
+        self._meta = metadata_list
+
+    def get(self, *, ids=None, where=None, limit=None, offset=None, include=None):
+        offset = offset or 0
+        limit = limit if limit is not None else len(self._meta)
+        page = self._meta[offset : offset + limit]
+        return {"ids": [], "documents": [], "metadatas": page}
+
+    def count(self):
+        return len(self._meta)
+
+
+@pytest.fixture
+def _stub_status_dependencies(monkeypatch):
+    """Stub out the chroma-only and palace-opening deps so status() is unit-testable.
+
+    - ``_sqlite_wing_room_counts`` returns None (non-chroma backend)
+    - ``hnsw_capacity_status`` returns a non-diverged dict
+    - ``_open_collection_or_explain`` returns the mock collection
+    """
+    monkeypatch.setattr(
+        "mempalace.backends.chroma._sqlite_wing_room_counts",
+        lambda palace_path, collection_name: None,
+    )
+    monkeypatch.setattr(
+        "mempalace.backends.chroma.hnsw_capacity_status",
+        lambda palace_path, collection_name: {"diverged": False, "status": "unknown"},
+    )
+    yield
+
+
+# ---------------------------------------------------------------------------
+# 1. Fast path: get_all_metadata() is used when present
+# ---------------------------------------------------------------------------
+
+
+class TestStatusUsesGetAllMetadata:
+    """miner.status() must delegate to get_all_metadata() when the collection
+    exposes it, avoiding the O(n^2) offset loop."""
+
+    def test_uses_get_all_metadata_when_present(self, _stub_status_dependencies, monkeypatch, capsys):
+        from mempalace import miner
+
+        meta = [
+            {"wing": "sessions", "room": "technical"},
+            {"wing": "sessions", "room": "technical"},
+            {"wing": "sessions", "room": "planning"},
+            {"wing": "knowledge", "room": "decisions"},
+        ]
+        col = _make_collection_with_get_all(meta)
+        monkeypatch.setattr(
+            miner, "_open_collection_or_explain", lambda palace_path: col
+        )
+
+        miner.status("/fake/palace")
+        out = capsys.readouterr().out
+
+        # Correct total
+        assert "4 drawers" in out
+        # Correct wing/room breakdown
+        assert "sessions" in out
+        assert "technical" in out
+        assert "planning" in out
+        assert "knowledge" in out
+
+    def test_does_not_call_offset_loop_when_get_all_present(
+        self, _stub_status_dependencies, monkeypatch, capsys
+    ):
+        """Regression guard: once get_all_metadata() is available, the offset
+        loop must NOT execute -- doing both would silently double the read cost."""
+        from mempalace import miner
+
+        col = _make_collection_with_get_all([{"wing": "a", "room": "b"}])
+        monkeypatch.setattr(
+            miner, "_open_collection_or_explain", lambda palace_path: col
+        )
+
+        miner.status("/fake/palace")
+        capsys.readouterr()  # drain
+
+        assert col.get_calls == [], (
+            f"get() offset loop must not run when get_all_metadata() is present, "
+            f"but got {len(col.get_calls)} calls"
+        )
+
+    def test_does_not_call_count_when_get_all_present(
+        self, _stub_status_dependencies, monkeypatch, capsys
+    ):
+        """count() is only needed by the fallback offset loop; the fast path
+        uses len(metas) instead."""
+        from mempalace import miner
+
+        col = _make_collection_with_get_all([{"wing": "a", "room": "b"}])
+        monkeypatch.setattr(
+            miner, "_open_collection_or_explain", lambda palace_path: col
+        )
+        # Spy on count() -- it should NOT be called
+        col.count = mock.MagicMock(side_effect=AssertionError("count() should not be called"))
+
+        miner.status("/fake/palace")
+        capsys.readouterr()
+
+        col.count.assert_not_called()
+
+    def test_handles_none_metadata_cells(self, _stub_status_dependencies, monkeypatch, capsys):
+        """Partially-flushed rows can have None metadata; status() must coerce
+        them to {} before calling .get() (same pattern as mcp_server)."""
+        from mempalace import miner
+
+        meta = [
+            {"wing": "sessions", "room": "technical"},
+            None,  # partially-flushed row
+            {"wing": "sessions", "room": "planning"},
+        ]
+        col = _make_collection_with_get_all(meta)
+        monkeypatch.setattr(
+            miner, "_open_collection_or_explain", lambda palace_path: col
+        )
+
+        miner.status("/fake/palace")
+        out = capsys.readouterr().out
+
+        assert "3 drawers" in out
+        assert "?" in out  # None metadata → wing "?" room "?"
+
+    def test_large_metadata_set_uses_single_pass(
+        self, _stub_status_dependencies, monkeypatch, capsys
+    ):
+        """With 10k+ drawers, get_all_metadata() must still make only ONE call
+        and NOT degrade to the offset loop."""
+        from mempalace import miner
+
+        meta = [{"wing": "sessions", "room": "technical"} for _ in range(10000)]
+        col = _make_collection_with_get_all(meta)
+        monkeypatch.setattr(
+            miner, "_open_collection_or_explain", lambda palace_path: col
+        )
+
+        miner.status("/fake/palace")
+        capsys.readouterr()
+
+        assert col.get_calls == [], "offset loop must not run for large collections"
+
+
+# ---------------------------------------------------------------------------
+# 2. Fallback: offset loop still works for legacy collections
+# ---------------------------------------------------------------------------
+
+
+class TestStatusFallbackOffsetLoop:
+    """When get_all_metadata() is NOT present, status() must fall back to the
+    offset-paginated col.get() loop with byte-for-byte the same behavior
+    as before the fix."""
+
+    def test_fallback_uses_offset_loop(self, _stub_status_dependencies, monkeypatch, capsys):
+        from mempalace import miner
+
+        meta = [{"wing": "a", "room": "x"}, {"wing": "b", "room": "y"}]
+        col = _LegacyCollection(meta)
+        assert not hasattr(col, "get_all_metadata")
+
+        monkeypatch.setattr(
+            miner, "_open_collection_or_explain", lambda palace_path: col
+        )
+
+        miner.status("/fake/palace")
+        out = capsys.readouterr().out
+
+        assert "2 drawers" in out
+        assert "a" in out
+        assert "b" in out
+
+    def test_fallback_empty_collection(self, _stub_status_dependencies, monkeypatch, capsys):
+        from mempalace import miner
+
+        col = _LegacyCollection([])
+        monkeypatch.setattr(
+            miner, "_open_collection_or_explain", lambda palace_path: col
+        )
+
+        miner.status("/fake/palace")
+        out = capsys.readouterr().out
+
+        assert "0 drawers" in out