Merge pull request #1702 from MemPalace/feat/migrate-wing-normalize

feat(migrate): mempalace migrate-wings — normalize legacy wing names (#1675 follow-up)

Author: igorls

docs/recovery/wing-name-migration.md

@@ -0,0 +1,63 @@
+# Recovery: legacy wing names split after the normalization change
+
+**Companion to #1675.** `normalize_wing_name` now strips leading and trailing
+separators, so a path-encoded project dir like `-home-user-proj` derives the
+wing `home_user_proj` instead of `_home_user_proj`. Palaces mined before that
+change filed drawers under the old, separator-padded name. New mining and diary
+writes land on the new name, so the two no longer meet — the history is
+**split**, not lost. `mempalace migrate-wings` re-unites them.
+
+## Symptom
+
+After upgrading, a project that used to surface its memories returns less than
+expected, and `mempalace status` shows two wings for one project — e.g. both
+`_home_user_proj` (old drawers) and `home_user_proj` (newly mined). MCP writes
+to the padded wing may also have been rejected, since `sanitize_name` does not
+accept a leading underscore.
+
+## Recovery
+
+Preview first — this never modifies anything:
+
+```bash
+mempalace migrate-wings --dry-run
+mempalace migrate-wings --dry-run --palace /path/to/palace
+```
+
+The plan lists each rename and flags collisions that will **merge** into an
+existing wing:
+
+```
+  Wing-name migration plan:
+    '_home_user_proj' -> 'home_user_proj': 1284 drawer(s), 96 closet(s)  (MERGE into existing wing)
+```
+
+Apply it:
+
+```bash
+mempalace migrate-wings           # prompts for confirmation
+mempalace migrate-wings --yes     # no prompt
+```
+
+## What it does
+
+- Re-keys the `wing` **metadata field** on drawers and closets to the normalized
+  form, merging collisions into the existing wing.
+- Re-keys the `topics_by_wing` registry (merging topic lists on collision).
+
+## What it leaves alone
+
+- **Drawer/closet IDs** are untouched. The wing in an ID (`drawer_<wing>_…`) is
+  an opaque prefix that is never decoded back into a wing, so leaving it keeps
+  closet `→drawer_id` pointers valid and lets future mining still skip
+  already-mined files (no duplicates). The verbatim drawer content is never
+  read or rewritten.
+- **Tunnels** already normalize wing names at read time, so they resolve under
+  the new name without a rewrite.
+
+## Notes
+
+- **Idempotent.** A second run reports "nothing to migrate" and changes nothing.
+- **Backend-agnostic.** Works on any configured storage backend.
+- Run it once per palace after upgrading. New palaces are born with normalized
+  wing names and never need it.

mempalace/cli.py

@@ -812,6 +812,18 @@ def cmd_migrate(args):
     )
 
 
+def cmd_migrate_wings(args):
+    """Normalize legacy wing names (strip leading/trailing separators)."""
+    palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
+    from .migrate import migrate_wing_names
+
+    migrate_wing_names(
+        palace_path=palace_path,
+        dry_run=args.dry_run,
+        confirm=getattr(args, "yes", False),
+    )
+
+
 def cmd_status(args):
     from .miner import status
 
@@ -1662,6 +1674,18 @@ def main():
         "--yes", action="store_true", help="Skip confirmation for destructive changes"
     )
 
+    # migrate-wings
+    p_migrate_wings = sub.add_parser(
+        "migrate-wings",
+        help="Normalize legacy wing names (strip leading/trailing separators) so pre-#1675 palaces stay discoverable",
+    )
+    p_migrate_wings.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would change without modifying the palace",
+    )
+    p_migrate_wings.add_argument("--yes", action="store_true", help="Skip the confirmation prompt")
+
     p_status = sub.add_parser("status", help="Show what's been filed")
     p_status.add_argument(
         "--backend",
@@ -1706,6 +1730,7 @@ def main():
         "repair": cmd_repair,
         "repair-status": cmd_repair_status,
         "migrate": cmd_migrate,
+        "migrate-wings": cmd_migrate_wings,
         "status": cmd_status,
     }
     dispatch[args.command](args)

mempalace/migrate.py

@@ -360,3 +360,218 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
 
     print(f"\n{'=' * 60}\n")
     return True
+
+
+# ---------------------------------------------------------------------------
+# Wing-name normalization migration (#1675 follow-up)
+# ---------------------------------------------------------------------------
+#
+# normalize_wing_name now strips leading/trailing separators, so a path-encoded
+# dirname like ``-home-user-proj`` derives ``home_user_proj`` instead of
+# ``_home_user_proj``. Palaces built before that rule filed drawers under the
+# old, leading-underscore wing, which the new derivation no longer matches —
+# searches and diary reads under the new name miss the old memories.
+#
+# This migration re-keys the ``wing`` metadata field on drawers and closets to
+# the normalized form, merging collisions. Drawer/closet IDs embed the wing as
+# an opaque prefix that is never decoded back into a wing (verified: nothing
+# splits a wing out of an ID; mining idempotency keys on ``source_file``), so
+# the IDs are left untouched — closet ``→drawer_id`` pointers stay valid and
+# future mining still skips already-mined files. Tunnels resolve via existing
+# read-time normalization and need no rewrite. The pass is idempotent.
+
+
+def _normalized_wing_target(wing):
+    """Return the normalized wing if it differs from ``wing``, else ``None``.
+
+    ``None`` means "no migration needed" — either the value is not a non-empty
+    string, normalization is a no-op, or it would normalize to empty.
+    """
+    from .config import normalize_wing_name
+
+    if not isinstance(wing, str) or not wing:
+        return None
+    # Apply the full normalization and explicitly strip leading/trailing
+    # separators. The strip is this migration's whole purpose (#1675); doing it
+    # here rather than relying on normalize_wing_name keeps the migration correct
+    # even when run against a build whose normalize_wing_name predates #1675, and
+    # matches the post-#1675 derivation exactly.
+    target = normalize_wing_name(wing).strip("_")
+    if not target or target == wing:
+        return None
+    return target
+
+
+def plan_wing_renames(items):
+    """Pure planner over ``(id, metadata)`` pairs.
+
+    Returns ``(summary, updates)`` where ``summary`` is ``{(old, new): count}``
+    and ``updates`` is ``[(id, new_metadata), ...]`` for only the records whose
+    wing changes. Metadata is copied; only the ``wing`` key is rewritten.
+    """
+    summary = defaultdict(int)
+    updates = []
+    for rec_id, meta in items:
+        meta = dict(meta or {})
+        target = _normalized_wing_target(meta.get("wing"))
+        if target is None:
+            continue
+        summary[(meta["wing"], target)] += 1
+        meta["wing"] = target
+        updates.append((rec_id, meta))
+    return summary, updates
+
+
+def _iter_collection_items(col, batch_size=1000):
+    """Yield ``(id, metadata)`` for every record in a backend collection."""
+    total = col.count()
+    offset = 0
+    while offset < total:
+        batch = col.get(limit=batch_size, offset=offset, include=["metadatas"])
+        ids = batch.ids if hasattr(batch, "ids") else batch["ids"]
+        metas = batch.metadatas if hasattr(batch, "metadatas") else batch["metadatas"]
+        if not ids:
+            break
+        for rec_id, meta in zip(ids, metas):
+            yield rec_id, meta
+        offset += len(ids)
+
+
+def _apply_wing_updates(col, updates, batch_size=500):
+    """Re-label the ``wing`` metadata field in place for the planned updates."""
+    for i in range(0, len(updates), batch_size):
+        chunk = updates[i : i + batch_size]
+        col.update(ids=[u[0] for u in chunk], metadatas=[u[1] for u in chunk])
+
+
+def _plan_topics_by_wing_renames():
+    """Return ``{old_wing: new_wing}`` for ``topics_by_wing`` keys to normalize."""
+    try:
+        from .miner import _load_known_entities_raw
+
+        reg = _load_known_entities_raw()
+    except Exception:
+        return {}
+    tbw = reg.get("topics_by_wing")
+    if not isinstance(tbw, dict):
+        return {}
+    renames = {}
+    for wing in list(tbw.keys()):
+        target = _normalized_wing_target(wing)
+        if target is not None:
+            renames[wing] = target
+    return renames
+
+
+def _apply_topics_by_wing_renames(renames):
+    """Re-key ``topics_by_wing`` in known_entities.json, merging on collision."""
+    if not renames:
+        return
+    import json
+
+    from .miner import _ENTITY_REGISTRY_PATH, _load_known_entities_raw
+
+    try:
+        reg = _load_known_entities_raw()
+    except Exception:
+        return
+    tbw = reg.get("topics_by_wing")
+    if not isinstance(tbw, dict):
+        return
+    for old, new in renames.items():
+        if old not in tbw:
+            continue
+        old_topics = tbw.pop(old) or []
+        if new in tbw:
+            merged = list(tbw[new])
+            for topic in old_topics:
+                if topic not in merged:
+                    merged.append(topic)
+            tbw[new] = merged
+        else:
+            tbw[new] = old_topics
+    reg["topics_by_wing"] = tbw
+    os.makedirs(os.path.dirname(_ENTITY_REGISTRY_PATH), exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(_ENTITY_REGISTRY_PATH), suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(reg, f, ensure_ascii=False, indent=2)
+        os.replace(tmp, _ENTITY_REGISTRY_PATH)
+    except Exception:
+        if os.path.exists(tmp):
+            os.remove(tmp)
+        raise
+
+
+def migrate_wing_names(palace_path: str, dry_run: bool = False, confirm: bool = False) -> bool:
+    """Normalize legacy wing names in ``palace_path`` (strip leading/trailing
+    separators), so palaces built before #1675 keep their memories discoverable.
+
+    Re-keys the ``wing`` metadata on drawers and closets in place (IDs untouched)
+    and the ``topics_by_wing`` registry, merging collisions. Idempotent.
+
+    Returns True if anything was (or, in dry-run, would be) migrated.
+    """
+    from .palace import get_closets_collection, get_collection
+
+    try:
+        drawers = get_collection(palace_path, create=False)
+    except Exception as exc:
+        print(f"  No drawer collection found at {palace_path} ({exc}).")
+        return False
+
+    d_items = list(_iter_collection_items(drawers))
+    all_wings = {(m or {}).get("wing") for _, m in d_items if (m or {}).get("wing")}
+    d_summary, d_updates = plan_wing_renames(d_items)
+
+    closets = None
+    c_summary, c_updates = defaultdict(int), []
+    try:
+        closets = get_closets_collection(palace_path, create=False)
+        c_summary, c_updates = plan_wing_renames(_iter_collection_items(closets))
+    except Exception:
+        closets = None
+
+    topic_renames = _plan_topics_by_wing_renames()
+
+    if not d_updates and not c_updates and not topic_renames:
+        print("  All wing names are already normalized — nothing to migrate.")
+        return False
+
+    print("\n  Wing-name migration plan:")
+    merged = defaultdict(lambda: [0, 0])
+    for key, count in d_summary.items():
+        merged[key][0] = count
+    for key, count in c_summary.items():
+        merged[key][1] = count
+    for (old, new), (d_count, c_count) in sorted(merged.items()):
+        note = "  (MERGE into existing wing)" if new in all_wings else ""
+        print(f"    {old!r} -> {new!r}: {d_count} drawer(s), {c_count} closet(s){note}")
+    if topic_renames:
+        print(f"    topics_by_wing: {len(topic_renames)} key(s) re-keyed")
+
+    if dry_run:
+        print("\n  DRY RUN — no changes made.\n")
+        return True
+
+    if not confirm:
+        try:
+            resp = input("  Apply this wing-name migration? [y/N] ").strip().lower()
+        except EOFError:
+            resp = ""
+        if resp not in ("y", "yes"):
+            print("  Aborted.")
+            return False
+
+    _apply_wing_updates(drawers, d_updates)
+    if closets is not None and c_updates:
+        _apply_wing_updates(closets, c_updates)
+    _apply_topics_by_wing_renames(topic_renames)
+
+    parts = [f"{len(d_updates)} drawer(s)"]
+    if c_updates:
+        parts.append(f"{len(c_updates)} closet(s)")
+    if topic_renames:
+        parts.append(f"{len(topic_renames)} topic key(s)")
+    print(f"\n  Migrated {', '.join(parts)}.\n")
+    return True